Merge pull request #6660 from escattone/add-spam-confidence-2326

add spam confidence

Author: akatsoulas

kitsune/llm/questions/prompt.py

@@ -23,11 +23,13 @@
   - It contains QR codes or images containing external links.
   - It is not relevant to Mozilla's "{product}" product.
 
-# Specific instructions
+# Instructions
 Given a question, you must do the following:
 - Determine whether or not the question is spam by considering each of the spam cases
   listed above.
 - Provide a reason for your determination.
+- Provide your level of confidence in the determination as an integer from 0 to 100, with
+  0 representing the lowest confidence and 100 the highest.
 
 # Response format instructions
 {format_instructions}
@@ -48,9 +50,10 @@
 {topics}
 ```
 
-# Specific instructions
-- Given a question, consider the title, description, and examples of each one of the
-  eligible topics listed above, and then select the most relevant topic.
+# Instructions
+Given a question, you must do the following:
+- Consider the title, description, and examples (if provided) of each one of the eligible
+  topics listed above, and then select the most relevant topic.
 - If you conclude that the question does not relate to Mozilla's "{product}" product,
   or does not provide enough information to make a selection, select the "Undefined" topic.
 - Provide a reason for your selection.
@@ -67,9 +70,18 @@
             type="bool",
             description="A boolean that when true indicates that the question is spam.",
         ),
+        ResponseSchema(
+            name="confidence",
+            type="int",
+            description=(
+                "An integer from 0 to 100 that indicates the level of confidence in the"
+                " determination of whether or not the question is spam, with 0 representing"
+                " the lowest confidence and 100 the highest."
+            ),
+        ),
         ResponseSchema(
             name="reason",
-            type="string",
+            type="str",
             description="The reason for identifying the question as spam or not spam.",
         ),
     )
@@ -80,19 +92,29 @@
     (
         ResponseSchema(
             name="topic",
-            type="string",
-            description="The topic selected for the question.",
+            type="str",
+            description="The title of the topic selected for the question.",
         ),
         ResponseSchema(
             name="reason",
-            type="string",
+            type="str",
             description="The reason for selecting the topic.",
         ),
     )
 )
 
 
-spam_prompt = ChatPromptTemplate((("system", SPAM_INSTRUCTIONS), ("human", "{question}")))
+spam_prompt = ChatPromptTemplate(
+    (
+        ("system", SPAM_INSTRUCTIONS),
+        ("human", "{question}"),
+    )
+).partial(format_instructions=spam_parser.get_format_instructions())
 
 
-topic_prompt = ChatPromptTemplate((("system", TOPIC_INSTRUCTIONS), ("human", "{question}")))
+topic_prompt = ChatPromptTemplate(
+    (
+        ("system", TOPIC_INSTRUCTIONS),
+        ("human", "{question}"),
+    )
+).partial(format_instructions=topic_parser.get_format_instructions())